{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import psutil\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 这里数据比较大 分批次读取 原本是CSV格式的文件 但是体量十分巨大 用自己本地的电脑硬是打不开 电脑卡住了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chunk: 0\n",
      "chunk: 1\n",
      "chunk: 2\n",
      "chunk: 3\n",
      "chunk: 4\n",
      "chunk: 5\n",
      "chunk: 6\n",
      "chunk: 7\n",
      "chunk: 8\n",
      "chunk: 9\n",
      "chunk: 10\n",
      "chunk: 11\n",
      "chunk: 12\n",
      "chunk: 13\n",
      "chunk: 14\n",
      "chunk: 15\n",
      "chunk: 16\n",
      "chunk: 17\n",
      "chunk: 18\n",
      "chunk: 19\n",
      "chunk: 20\n",
      "chunk: 21\n",
      "chunk: 22\n",
      "chunk: 23\n",
      "chunk: 24\n",
      "chunk: 25\n",
      "chunk: 26\n",
      "chunk: 27\n",
      "chunk: 28\n",
      "chunk: 29\n",
      "chunk: 30\n",
      "chunk: 31\n",
      "chunk: 32\n",
      "chunk: 33\n",
      "chunk: 34\n",
      "chunk: 35\n",
      "chunk: 36\n",
      "chunk: 37\n",
      "chunk: 38\n",
      "chunk: 39\n",
      "chunk: 40\n",
      "chunk: 41\n",
      "chunk: 42\n",
      "chunk: 43\n",
      "chunk: 44\n",
      "chunk: 45\n",
      "chunk: 46\n",
      "chunk: 47\n",
      "chunk: 48\n",
      "chunk: 49\n",
      "chunk: 50\n",
      "chunk: 51\n",
      "chunk: 52\n",
      "chunk: 53\n",
      "chunk: 54\n",
      "chunk: 55\n",
      "chunk: 56\n",
      "chunk: 57\n",
      "chunk: 58\n",
      "chunk: 59\n",
      "chunk: 60\n",
      "chunk: 61\n",
      "chunk: 62\n",
      "chunk: 63\n",
      "chunk: 64\n",
      "chunk: 65\n",
      "chunk: 66\n",
      "chunk: 67\n",
      "chunk: 68\n",
      "chunk: 69\n",
      "chunk: 70\n",
      "chunk: 71\n",
      "chunk: 72\n",
      "chunk: 73\n",
      "chunk: 74\n",
      "chunk: 75\n",
      "chunk: 76\n",
      "chunk: 77\n",
      "chunk: 78\n",
      "chunk: 79\n",
      "chunk: 80\n",
      "chunk: 81\n",
      "chunk: 82\n",
      "chunk: 83\n",
      "chunk: 84\n",
      "chunk: 85\n",
      "chunk: 86\n",
      "chunk: 87\n",
      "chunk: 88\n",
      "chunk: 89\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrLoadDll</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806688</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806689</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806690</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806691</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806692</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtTerminateProcess</td>\n",
       "      <td>2336</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>89806693 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  label                     api   tid  index\n",
       "0               1      5              LdrLoadDll  2488      0\n",
       "1               1      5  LdrGetProcedureAddress  2488      1\n",
       "2               1      5  LdrGetProcedureAddress  2488      2\n",
       "3               1      5  LdrGetProcedureAddress  2488      3\n",
       "4               1      5  LdrGetProcedureAddress  2488      4\n",
       "...           ...    ...                     ...   ...    ...\n",
       "89806688    13887      2                 NtClose  2336    618\n",
       "89806689    13887      2                 NtClose  2336    619\n",
       "89806690    13887      2                 NtClose  2336    620\n",
       "89806691    13887      2                 NtClose  2336    621\n",
       "89806692    13887      2      NtTerminateProcess  2336    622\n",
       "\n",
       "[89806693 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据加载\n",
    "def get_data(filename):\n",
    "    result=[]\n",
    "    chunk_index=0\n",
    "    for df in pd.read_csv(open(filename,'r'),chunksize=1000000):\n",
    "        result.append(df)\n",
    "        print(\"chunk:\",chunk_index)\n",
    "        chunk_index+=1\n",
    "    result=pd.concat(result,ignore_index=True,axis=0)\n",
    "    return result\n",
    "\n",
    "#获取全量数据\n",
    "train=get_data('./security_train.csv')\n",
    "train#数据比较大 快9千万"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chunk: 0\n",
      "chunk: 1\n",
      "chunk: 2\n",
      "chunk: 3\n",
      "chunk: 4\n",
      "chunk: 5\n",
      "chunk: 6\n",
      "chunk: 7\n",
      "chunk: 8\n",
      "chunk: 9\n",
      "chunk: 10\n",
      "chunk: 11\n",
      "chunk: 12\n",
      "chunk: 13\n",
      "chunk: 14\n",
      "chunk: 15\n",
      "chunk: 16\n",
      "chunk: 17\n",
      "chunk: 18\n",
      "chunk: 19\n",
      "chunk: 20\n",
      "chunk: 21\n",
      "chunk: 22\n",
      "chunk: 23\n",
      "chunk: 24\n",
      "chunk: 25\n",
      "chunk: 26\n",
      "chunk: 27\n",
      "chunk: 28\n",
      "chunk: 29\n",
      "chunk: 30\n",
      "chunk: 31\n",
      "chunk: 32\n",
      "chunk: 33\n",
      "chunk: 34\n",
      "chunk: 35\n",
      "chunk: 36\n",
      "chunk: 37\n",
      "chunk: 38\n",
      "chunk: 39\n",
      "chunk: 40\n",
      "chunk: 41\n",
      "chunk: 42\n",
      "chunk: 43\n",
      "chunk: 44\n",
      "chunk: 45\n",
      "chunk: 46\n",
      "chunk: 47\n",
      "chunk: 48\n",
      "chunk: 49\n",
      "chunk: 50\n",
      "chunk: 51\n",
      "chunk: 52\n",
      "chunk: 53\n",
      "chunk: 54\n",
      "chunk: 55\n",
      "chunk: 56\n",
      "chunk: 57\n",
      "chunk: 58\n",
      "chunk: 59\n",
      "chunk: 60\n",
      "chunk: 61\n",
      "chunk: 62\n",
      "chunk: 63\n",
      "chunk: 64\n",
      "chunk: 65\n",
      "chunk: 66\n",
      "chunk: 67\n",
      "chunk: 68\n",
      "chunk: 69\n",
      "chunk: 70\n",
      "chunk: 71\n",
      "chunk: 72\n",
      "chunk: 73\n",
      "chunk: 74\n",
      "chunk: 75\n",
      "chunk: 76\n",
      "chunk: 77\n",
      "chunk: 78\n",
      "chunk: 79\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>RegOpenKeyExA</td>\n",
       "      <td>2332</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>CopyFileA</td>\n",
       "      <td>2332</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>OpenSCManagerA</td>\n",
       "      <td>2332</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>CreateServiceA</td>\n",
       "      <td>2332</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>RegOpenKeyExA</td>\n",
       "      <td>2468</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288370</th>\n",
       "      <td>12955</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288371</th>\n",
       "      <td>12955</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288372</th>\n",
       "      <td>12955</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2740</td>\n",
       "      <td>1448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288373</th>\n",
       "      <td>12955</td>\n",
       "      <td>__exception__</td>\n",
       "      <td>2740</td>\n",
       "      <td>1449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288374</th>\n",
       "      <td>12955</td>\n",
       "      <td>NtTerminateProcess</td>\n",
       "      <td>2740</td>\n",
       "      <td>1450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79288375 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id                 api   tid  index\n",
       "0               1       RegOpenKeyExA  2332      0\n",
       "1               1           CopyFileA  2332      1\n",
       "2               1      OpenSCManagerA  2332      2\n",
       "3               1      CreateServiceA  2332      3\n",
       "4               1       RegOpenKeyExA  2468      0\n",
       "...           ...                 ...   ...    ...\n",
       "79288370    12955        Thread32Next  2740   1446\n",
       "79288371    12955        Thread32Next  2740   1447\n",
       "79288372    12955             NtClose  2740   1448\n",
       "79288373    12955       __exception__  2740   1449\n",
       "79288374    12955  NtTerminateProcess  2740   1450\n",
       "\n",
       "[79288375 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取全量数据\n",
    "test=get_data('./security_test.csv')\n",
    "test#数据比较大 快8千万"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 内存监控"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "总内存： 257785.796875\n",
      "已使用内存： 143526.984375\n",
      "空闲内存： 28959.3671875\n",
      "使用占比： 80.7\n",
      "当前线程PID： 487321\n"
     ]
    }
   ],
   "source": [
    "mem = psutil.virtual_memory()\n",
    "print('总内存：',mem.total/1024/1024)\n",
    "print('已使用内存：', mem.used/1024/1024)\n",
    "print('空闲内存：', mem.free/1024/1024)\n",
    "print('使用占比：',mem.percent)\n",
    "print('当前线程PID：', os.getpid())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 保存为 pickle文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#把大文件转成为小文件试试，pickle就是管理python对象 可以用很快的方式去完成读取\n",
    "import pickle#wb代表写也就是存储文件 rb就是读文件\n",
    "with open('./train.pkl','wb') as f:\n",
    "    pickle.dump(train,f)\n",
    "\n",
    "with open('./test.pkl','wb') as f:\n",
    "    pickle.dump(test,f)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 这个程序先到此为止  就先把大文件转成pickle文件 下一步再从读取pickle文件开始进行下一步"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
